# -*- coding:utf-8 -*-

import module.storage
import module.ksw
from multiprocessing.dummy import Pool as ThreadPool  # 引入多线程
import binascii

class Main():
    ksw = module.ksw.Ksw()
    story = module.storage.Storage()
    pool = ThreadPool()

    def __init__(self):
        self.complete = []

    def main(self, url):
        list = self.ksw.get_list(url)
        self.pool.map(self.handler, list)
        self.pool.close()
        self.pool.join()

        for i in list:
            self.handler(i)

        if self.story.get_mysql(self.complete):
            print("全部章节存入SQL成功！！！！！")
        else:
            print("全部章节存入SQL失败！！")

    def handler(self, url):
        info = self.ksw.get_story(url)
        flag = binascii.crc32(info['content'].encode()) #随机数组
        with open('./resource/%s.txt' % flag, 'w', encoding = 'utf8') as f:
            f.write(info['content'])
        self.complete.append((info['title'], flag))
        print("%s-------------爬取成功！" % (info['title']))


# 重构结构，单线程爬取完成
if __name__ == "__main__":
    g = Main()
    g.main('https://www.00ksw.com/html/51/51193/')
